
<!DOCTYPE html>

<html xmlns="http://www.w3.org/1999/xhtml">
  <head>
    <meta charset="utf-8" />
    <title>UCTB.utils.multi_threads &#8212; UCTB  documentation</title>
    <link rel="stylesheet" href="../../../_static/nature.css" type="text/css" />
    <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
    <script type="text/javascript" id="documentation_options" data-url_root="../../../" src="../../../_static/documentation_options.js"></script>
    <script type="text/javascript" src="../../../_static/jquery.js"></script>
    <script type="text/javascript" src="../../../_static/underscore.js"></script>
    <script type="text/javascript" src="../../../_static/doctools.js"></script>
    <script type="text/javascript" src="../../../_static/language_data.js"></script>
    <link rel="index" title="Index" href="../../../genindex.html" />
    <link rel="search" title="Search" href="../../../search.html" /> 
  </head><body>
    <div class="related" role="navigation" aria-label="related navigation">
      <h3>Navigation</h3>
      <ul>
        <li class="right" style="margin-right: 10px">
          <a href="../../../genindex.html" title="General Index"
             accesskey="I">index</a></li>
        <li class="right" >
          <a href="../../../py-modindex.html" title="Python Module Index"
             >modules</a> |</li>
        <li class="nav-item nav-item-0"><a href="../../../index.html">UCTB  documentation</a> &#187;</li>
          <li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li> 
      </ul>
    </div>  

    <div class="document">
      <div class="documentwrapper">
        <div class="bodywrapper">
          <div class="body" role="main">
            
  <h1>Source code for UCTB.utils.multi_threads</h1><div class="highlight"><pre>
<span></span><span class="kn">import</span> <span class="nn">os</span>

<span class="kn">from</span> <span class="nn">multiprocessing</span> <span class="k">import</span> <span class="n">Pool</span><span class="p">,</span> <span class="n">Manager</span>
<span class="kn">from</span> <span class="nn">functools</span> <span class="k">import</span> <span class="n">reduce</span>


<span class="c1"># (my_rank, n_jobs, dataList, resultHandleFunction, parameterList)</span>
<div class="viewcode-block" id="multiple_process"><a class="viewcode-back" href="../../../UCTB.utils.html#UCTB.utils.multi_threads.multiple_process">[docs]</a><span class="k">def</span> <span class="nf">multiple_process</span><span class="p">(</span><span class="n">distribute_list</span><span class="p">,</span> <span class="n">partition_func</span><span class="p">,</span> <span class="n">task_func</span><span class="p">,</span> <span class="n">n_jobs</span><span class="p">,</span> <span class="n">reduce_func</span><span class="p">,</span> <span class="n">parameters</span><span class="p">):</span>
    <span class="k">if</span> <span class="n">callable</span><span class="p">(</span><span class="n">partition_func</span><span class="p">)</span> <span class="ow">and</span> <span class="n">callable</span><span class="p">(</span><span class="n">task_func</span><span class="p">)</span> <span class="ow">and</span> <span class="n">callable</span><span class="p">(</span><span class="n">reduce_func</span><span class="p">):</span>
        <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Parent process </span><span class="si">%s</span><span class="s1">.&#39;</span> <span class="o">%</span> <span class="n">os</span><span class="o">.</span><span class="n">getpid</span><span class="p">())</span>

        <span class="n">manager</span> <span class="o">=</span> <span class="n">Manager</span><span class="p">()</span>
        <span class="n">share_queue</span> <span class="o">=</span> <span class="n">manager</span><span class="o">.</span><span class="n">Queue</span><span class="p">()</span>
        <span class="n">locker</span> <span class="o">=</span> <span class="n">manager</span><span class="o">.</span><span class="n">Lock</span><span class="p">()</span>

        <span class="n">p</span> <span class="o">=</span> <span class="n">Pool</span><span class="p">()</span>
        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_jobs</span><span class="p">):</span>
            <span class="n">p</span><span class="o">.</span><span class="n">apply_async</span><span class="p">(</span><span class="n">task_func</span><span class="p">,</span> <span class="n">args</span><span class="o">=</span><span class="p">(</span><span class="n">share_queue</span><span class="p">,</span> <span class="n">locker</span><span class="p">,</span> <span class="n">partition_func</span><span class="p">(</span><span class="n">distribute_list</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="n">n_jobs</span><span class="p">),</span>
                                           <span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">+</span> <span class="n">parameters</span><span class="p">,))</span>
        <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Waiting for all sub_processes done...&#39;</span><span class="p">)</span>
        <span class="n">p</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
        <span class="n">p</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>
        <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;All sub_processes done.&#39;</span><span class="p">)</span>

        <span class="n">result_list</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="k">while</span> <span class="ow">not</span> <span class="n">share_queue</span><span class="o">.</span><span class="n">empty</span><span class="p">():</span>
            <span class="n">result_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">share_queue</span><span class="o">.</span><span class="n">get_nowait</span><span class="p">())</span>

        <span class="k">return</span> <span class="n">reduce</span><span class="p">(</span><span class="n">reduce_func</span><span class="p">,</span> <span class="n">result_list</span><span class="p">)</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Parameter error&#39;</span><span class="p">)</span></div>


<span class="c1"># Example</span>
<div class="viewcode-block" id="task"><a class="viewcode-back" href="../../../UCTB.utils.html#UCTB.utils.multi_threads.task">[docs]</a><span class="k">def</span> <span class="nf">task</span><span class="p">(</span><span class="n">share_queue</span><span class="p">,</span> <span class="n">locker</span><span class="p">,</span> <span class="n">data</span><span class="p">,</span> <span class="n">parameters</span><span class="p">):</span>

    <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Child process </span><span class="si">%s</span><span class="s1"> with pid </span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="p">(</span><span class="n">parameters</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">os</span><span class="o">.</span><span class="n">getpid</span><span class="p">()))</span>

    <span class="n">result</span> <span class="o">=</span> <span class="nb">sum</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>

    <span class="n">locker</span><span class="o">.</span><span class="n">acquire</span><span class="p">()</span>
    <span class="n">share_queue</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="n">result</span><span class="p">)</span>
    <span class="n">locker</span><span class="o">.</span><span class="n">release</span><span class="p">()</span></div>


<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">&quot;__main__&quot;</span><span class="p">:</span>

    <span class="n">data</span> <span class="o">=</span> <span class="p">[</span><span class="n">e</span> <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1000000</span><span class="p">)]</span>

    <span class="n">n_job</span> <span class="o">=</span> <span class="mi">4</span>

    <span class="n">sum_result</span> <span class="o">=</span> \
        <span class="n">multiple_process</span><span class="p">(</span><span class="n">distribute_list</span><span class="o">=</span><span class="n">data</span><span class="p">,</span>
                         <span class="n">partition_func</span><span class="o">=</span><span class="k">lambda</span> <span class="n">data</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="n">n_job</span><span class="p">:</span> <span class="p">[</span><span class="n">data</span><span class="p">[</span><span class="n">e</span><span class="p">]</span> <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">data</span><span class="p">))</span> <span class="k">if</span> <span class="n">e</span> <span class="o">%</span> <span class="n">n_job</span> <span class="o">==</span> <span class="n">i</span><span class="p">],</span>
                         <span class="n">task_func</span><span class="o">=</span><span class="n">task</span><span class="p">,</span> <span class="n">n_jobs</span><span class="o">=</span><span class="n">n_job</span><span class="p">,</span> <span class="n">reduce_func</span><span class="o">=</span><span class="k">lambda</span> <span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">:</span> <span class="n">x</span> <span class="o">+</span> <span class="n">y</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="p">[])</span>

    <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Result&#39;</span><span class="p">,</span> <span class="n">sum_result</span><span class="p">)</span>


</pre></div>

          </div>
        </div>
      </div>
      <div class="sphinxsidebar" role="navigation" aria-label="main navigation">
        <div class="sphinxsidebarwrapper">
<div id="searchbox" style="display: none" role="search">
  <h3 id="searchlabel">Quick search</h3>
    <div class="searchformwrapper">
    <form class="search" action="../../../search.html" method="get">
      <input type="text" name="q" aria-labelledby="searchlabel" />
      <input type="submit" value="Go" />
    </form>
    </div>
</div>
<script type="text/javascript">$('#searchbox').show(0);</script>
        </div>
      </div>
      <div class="clearer"></div>
    </div>
    <div class="related" role="navigation" aria-label="related navigation">
      <h3>Navigation</h3>
      <ul>
        <li class="right" style="margin-right: 10px">
          <a href="../../../genindex.html" title="General Index"
             >index</a></li>
        <li class="right" >
          <a href="../../../py-modindex.html" title="Python Module Index"
             >modules</a> |</li>
        <li class="nav-item nav-item-0"><a href="../../../index.html">UCTB  documentation</a> &#187;</li>
          <li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li> 
      </ul>
    </div>
    <div class="footer" role="contentinfo">
        &#169; Copyright 2019, Di Chai, Leye Wang, Jin Xu, Wenjie Yang, Liyue Chen.
      Created using <a href="http://sphinx-doc.org/">Sphinx</a> 2.2.1.
    </div>
  </body>
</html>